# coding:utf-8

import bs4
from scrapy.contrib.spiders import CrawlSpider
from ..items import LvjuItem
import scrapy


class LvjuSpider(CrawlSpider):
    name = "lvjuspider"
    allowed_domains = ["hainanfz.com"]
    start_urls = ["http://www.hainanfz.com/house/"]

    def parse(self, response):
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')
        citybox = soup.find_all('div', class_='rollBox')[0]
        for child in citybox.findAll('a'):
            city = child.string
            city_url = str(child.get("href"))
            citys = {
                'website': '旅居网', 'web_url': 'hainanfz.com',
                'city': city, 'city_id': city_url.replace('http://', '').replace('.hainanfz.com/house/', '')

            }
            yield scrapy.Request(city_url,callback=self.parse_cityPage_url,meta=citys)

    def parse_cityPage_url(self, response):
        meta = response.meta
        site = meta['city_id']
        city = meta['city']
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')
        page_num = 1
        if city == '琼中' or city == '昌江' or city == '白沙':
            pass
        else:
            page_num = int(soup.select(" .digg > a")[-2].get_text())

        for n in range(page_num):
            site_url = 'http://' + site + '.hainanfz.com/house/y' + str(n + 1) + '.html'
            yield scrapy.Request(site_url,callback=self.parse_city_estate, meta=meta)

    def parse_city_estate(self, response):
        meta = response.meta
        data = response.body
        soup = bs4.BeautifulSoup(data, 'lxml')
        estates = soup.select(".hiLeft > a")

        for child in estates:
            item = LvjuItem()
            estate_url = child.get('href')
            estate_id = child.get('href').replace('http://www.hainanfz.com/house/', '').replace('/', '')
            estate = child.get('title')
            meta['estate'] = estate
            meta['estate_id'] = estate_id
            item['website'] = meta['website']
            item['web_url'] = meta['web_url']
            item['city'] = meta['city']
            item['area'] = ''
            item['city_id'] = meta['city_id']
            item['estate'] = meta['estate']
            item['estate_id'] = meta['estate_id']
            item['estate_url'] = estate_url
            yield item


